﻿#coding=UTF-8
import datetime
import re
import HTMLParser
import logging
log = logging.getLogger("dataparser")

class DataParser(HTMLParser.HTMLParser):
  tags = ('table', 'th', 'tr', 'td', 'caption', 'thead', 'tbody', 'tfoot')
  patterns = ['table/caption',
              'table/tr/th',
              'table/thead/tr/th',
              'table/tr/td',
              'table/tbody/tr/td',
              'table/tfoot/tr/td']
  
  def __init__(self):
    self.records = []
    self.tableStack = []
    self.reset2()
    HTMLParser.HTMLParser.__init__(self)
    
  def unknown_decl(self, data):
    return
            
  def handle_data(self, data):
    if "/".join(self.tagStack) not in self.patterns \
    or 0 == len(data.strip()):
      return
    self.data += data.strip()
        
  def reset(self):
    HTMLParser.HTMLParser.reset(self)
    self.records = []
    self.tableStack = []
    self.date = (datetime.datetime.now(), "%Y%m%d")
    self.dates = ({}, {}) #Row, Col
    self.unit = ""
    self.units = ({}, {}) #Row, Col
    self.globalTags = []
    self.reset2()
    
  def reset2(self):
    self.tagStack = []
    self.data = ""
    self.currCol = 0
    self.currRow = 0
    self.colSpan = 1
    self.rowSpan = 1
    self.colTags = {}
    self.rowTags = {}
    self.cells = {}
    self.tagLevel = None
    self.tagLevels = [TagLevel1(), TagLevel2(), TagLevel3()]
        
  def handle_starttag(self, tag, attrs):
    if tag not in self.tags:
      return
    
    if 'table' == tag \
    and len(self.tagStack)>0:
      self.tableStack.append((self.tagStack, self.colTags, self.rowTags,
                              self.currCol, self.currRow, self.colSpan,
                              self.rowSpan, self.cells, self.tagLevel))
      self.reset2()
      
    self.tagStack.append(tag)
    if 'tr' == tag:
      self.currCol = 0
    
    if 'td' != tag \
    and 'th' != tag:
      return
    dAttrs = dict(attrs)
    if dAttrs.has_key('colspan'):
      self.colSpan = int(dAttrs['colspan'])
    else:
      self.colSpan = 1
    if dAttrs.has_key('rowspan'):
      self.rowSpan = int(dAttrs['rowspan'])
    else:
      self.rowSpan = 1
    while "Merged" == self.fillCell():
      self.currCol+=1
            
  def handle_endtag(self, tag):
    if tag not in self.tags \
    or 0 == len(self.tagStack) \
    or tag != self.tagStack[ - 1]:
      return
    
    self.tagStack.pop()
    if 'table' == tag \
    and len(self.tableStack)>0:
      prevTb = self.tableStack.pop()
      self.tagStack = prevTb[0]
      self.colTags = prevTb[1]
      self.rowTags = prevTb[2]
      self.currCol = prevTb[3]
      self.currRow = prevTb[4]
      self.colSpan = prevTb[5]
      self.rowSpan = prevTb[6]
      self.cells = prevTb[7]
      self.tagLevel = prevTb[8]

    data = [re.sub('\s', '', self.data).strip()]
    #TODO: isDate could always be False?
    if ('td' == tag \
    or 'th' == tag) \
    and "Single" == self.cells[self.currRow][self.currCol] \
    and (False == self.isDate(data) \
         or len(data[0].strip())>0) \
    and False == self.isUnit(data) \
    and len(data[0].strip())>0:
      rec = [0, ""]
      if True == self.isRecord(data[0], rec):
        tags = []
        if len(self.colTags[self.currCol]) > 0:
          tags += dict(self.colTags[self.currCol]).keys()
        if len(self.rowTags[self.currRow]) > 0:
          tags += dict(self.rowTags[self.currRow]).keys()
        if 0 == len(tags):
          log.warn("Record[%s] has no tags, ignored.", data[0])
        tags += self.globalTags
        self.records.append((rec[0], rec[1], self.getDate(), tags))          
      else:
        if 0 == len(self.dates[0]):
          if self.tagLevel is None:
            for tagLevel in self.tagLevels:
              if True == tagLevel.check(self, data):
                self.tagLevel = tagLevel
                break
          else:
            self.tagLevel.check(self, data)        
        if None != data[0] and len(data[0].strip())>0:
          self.addTags(self.colTags, self.currCol, data, self.colSpan, True)
          self.addTags(self.rowTags, self.currRow, data, self.rowSpan)

    if 'tr' == tag:
      self.currRow += 1
      if False == self.rowTags.has_key(self.currRow):
        self.rowTags[self.currRow] = []
    if ('td' == tag \
    or 'th' == tag):
      self.currCol += 1
    self.data = ""
  
  def isRecord(self, data, rec):
    if True == data.isdigit() \
    or (1 == data.count('.') \
    and True == data.replace('.', '').isdigit()):
      rec[0] = float(data)
      rec[1] = self.getUnit()
      return True
    if True == data[1:].isdigit() \
    or (1 == data[1:].count('.') \
    and True == data[1:].replace('.', '').isdigit()):
      rec[0] = float(data[1:])
      if '-' == data[0]:
        rec[0] *= -1
        rec[1] = self.getUnit()
      elif True == self.isUnit([data[0]], True):
        rec[1] = data[0]
      else: return False
      return True
    if True == data[0:len(data)-1].isdigit() \
    or (1 == data[0:len(data)-1].count('.') \
    and True == data[0:len(data)-1].replace('.', '').isdigit()):
      rec[0] = float(data[0:len(data)-1])
      rec[1] = data[-1]
      return True
    return False
  
  def addTag(self, tags, index, data, span, append=False):
    self.refineTagSymbol(data)
    if False == tags.has_key(index):
      tags[index] = []
    if True == append \
    and 1 == span \
    and len(tags[index])>0:
      last = tags[index].pop()
      if 1 == last[1][0] \
      and 0 != last[1][1]:
        data[0] = last[0] + data[0]
        self.refineTagSymbol(data)
      else:
        tags[index].append(last)
    #TODO: check duplicates?
    tags[index].append((data[0], (span, index)))
    
  def addTags(self, tags, index, data, span, append=False):
    self.addTag(tags, index, data, span, append)
    if span > 1:
      i = 1
      while i < span:
        self.addTag(tags, index+i, data, span)
        i+=1
        
  def refineTagSymbol(self, data):
    if re.match(u"(\d+-)*\d+米.*", data[0]) is not None:
      return
    
    rTag = [u"\([年|月].*\)",
            u"[。≥≤]",
            u"本表.*",
            u"[\s\d\-\(\).#%=a-zA-Z:]",
            ]
    for r in rTag:
      data[0] = re.sub(r, "", data[0])
  
  def fillCell(self):
    if True == self.cells.has_key(self.currRow) \
    and True == self.cells[self.currRow].has_key(self.currCol):
      assert "Merged" == self.cells[self.currRow][self.currCol]
      return "Merged"
    i = 0
    while i<self.rowSpan:
      j = 0
      while j<self.colSpan:
        if False == self.cells.has_key(self.currRow+i):
          self.cells[self.currRow+i] = {}
        self.cells[self.currRow+i][self.currCol+j] = "Merged"
        j+=1
      i+=1
    self.cells[self.currRow][self.currCol] = "Single"
    return "Single"
  
  def getDate(self):
    if True == self.dates[0].has_key(self.currRow):
      return self.dates[0][self.currRow]
    if True == self.dates[1].has_key(self.currCol):
      return self.dates[1][self.currCol]
    return self.date

  def isDate(self, data):
    rY = "19[7-9][0-9]|20[0-2][0-9]"
    rM = "1[0-2]|0?[1-9]"
    rD = "[1-2][0-9]|3[0-1]|0?[1-9]"
    uY = u"[年]"
    uM = u"[月]"
    uD = u"[日|号]"
    prefix = ".*[^\-\+0-9]+"
    rYear = [prefix + "(" + rY + ")"+uY+"(" + rM + ")"+uM+"(" + rD + ")"+uD,
             prefix + "(" + rY + ")[.-](" + rM + ")[.-](" + rD + ")",
             prefix + "(" + rY + ")"+uY+"(" + rM + ")"+uM,
             prefix + "(" + rY + ")[.-](" + rM + ")",
             prefix + "(" + rY + ")" +uY,
             prefix + "(" + rY + ")$",
             "(" + rY + ")"+uY+"(" + rM + ")"+uM+"(" + rD + ")"+uD,
             "(" + rY + ")[.-](" + rM + ")[.-](" + rD + ")",
             "(" + rY + ")"+uY+"(" + rM + ")"+uM,
             "(" + rY + ")[.-](" + rM + "$)",
             "(" + rY + ")" +uY,
             "(" + rY + ")$",
             ]
    rMonth = [prefix + "(" + rM + ")" + uM + "("+ rD + ")" +uD,
#              prefix + "(" + rM + ")[.-](" + rD + ")",
              prefix + "(" + rM + ")" +uM,
              "(" + rM + ")" + uM + "("+ rD + ")" +uD,
#              "(" + rM + ")[.-](" + rD + ")",
              "(" + rM + ")" +uM,
              ]
    rDay = [prefix + "(" + rD +")" +uD,
            "(" + rD +")" +uD
            ]
    yFmt = {1:"%Y",
            2:"%Y%m",
            3:"%Y%m%d"}
    mFmt = {1:"%Y%m",
            2:"%Y%m%d"}
    dFmt = {1:"%Y%m%d"}
    if True == self.checkDate(rYear, data, "", yFmt)  \
    or True == self.checkDate(rMonth, data, self.date[0].strftime("%Y"), mFmt) \
    or True == self.checkDate(rDay, data, self.date[0].strftime("%Y%m"), dFmt):
      data[0] = re.sub(data[1], self.replDate, data[0])
      return True
    return False
  
  def replDate(self, m):
    i = 1
    s = m.string[0:m.start(i)]
    while i <= len(m.groups()):
      if i+1 <= len(m.groups()):
        j = m.start(i+1)
      else:
        j = m.end(0)
      s = s + m.string[m.end(i):j]
      i += 1
    return s

  def checkDate(self, rDate, data, prefix, fmt):
    for r in rDate:
      m = re.match(r, data[0])
      if None != m:
        data.append(r)
        self.date = (datetime.datetime.strptime(prefix+"".join(m.groups()), fmt[len(m.groups())]), fmt[len(m.groups())])
        if 0 == self.currCol \
        and 1 == self.colSpan:
          self.dates[0][self.currRow] = (datetime.datetime.strptime(prefix+"".join(m.groups()), fmt[len(m.groups())]), fmt[len(m.groups())])
          return True
        if len(self.dates[0])>0:
          return False
        if True == self.dates[1].has_key(self.currCol):
          return False
        self.dates[1][self.currCol] = (datetime.datetime.strptime(prefix+"".join(m.groups()), fmt[len(m.groups())]), fmt[len(m.groups())])
        return True
    return False
  
  def getUnit(self):
    if len(self.dates[0]) > 0:
      if True == self.units[1].has_key(self.currCol):
        return self.units[1][self.currCol]
    if len(self.dates[1]) > 0:
      if True == self.units[0].has_key(self.currRow):
        return self.units[0][self.currRow]
    
    if True == self.units[1].has_key(self.currCol):
      return self.units[1][self.currCol]
    if True == self.units[0].has_key(self.currRow):
      return self.units[0][self.currRow]
    if True == self.units[1].has_key(self.currCol-1):
      self.units[1][self.currCol] = self.units[1][self.currCol-1]
      return self.units[1][self.currCol]
    if True == self.units[0].has_key(self.currRow-1):
      self.units[0][self.currRow] = self.units[0][self.currRow-1]
      return self.units[0][self.currRow]
    return self.unit

  def replUnit(self, m):
    return m.string[0:m.start(1)] + m.string[m.end(1):m.end(0)] 

  def isUnit(self, data, single=False):
    rUnitGlobal = [
             u"\D*单位[：|:](.*)",
             ]
    rUnit = [
             u"\D*(\(\D*?[%|\uff05|‰|元|个|所|座|吨|米|时|人|对|台|户|册|部|张|里|辆|起|度|顷|瓦|斤]\D*?\))",
             u"\D*(\(\D*?=100\D*?\))",
             u"(\D*?=1\d*\D*?)",
             u".*(\([0-9]*年＝100\))",
             u"\D*(\(.*?[吨]\D*?\))",
             ]
    
    for r in rUnitGlobal+rUnit:
      m = re.match(r, data[0])
      if None != m:
        if False == single:
          u = re.sub("[\(|\)]", "", m.groups()[0])
          if r in rUnitGlobal:
            self.unit = u
          self.units[0][self.currRow] = u
          self.units[1][self.currCol]= u
          if r in rUnit:
            data[0] = re.sub(r, self.replUnit, data[0])
            return False
        return True
    return False
  
class TagLevel(object):
  def __init__(self):
    self.tagStack = [] #[(tag, level, level-1.index)]
    
  def check(self, dataparser, data):
    return False
  
class TagLevel1(TagLevel):
  def check(self, dataparser, data):
    if dataparser.currCol > 0: return False
    
    rLevel = [u"([一|二|三|四|五|六|七|八|九|十]+\.)", #level 0
              u"([A-Z]+\.)", #level 1
              u"([a-z]+\.)", #level 2
              u"([0-9]+(\.[0-9])+)", #level 4+
              u"([0-9]+\.)", #level 3
              ]
    level = 99
    for r in rLevel:
      m = re.match(r, data[0])
      if None != m:
        if 3 == rLevel.index(r):
          level = m.groups()[0].count(".") + 1
        elif 4 == rLevel.index(r):
          level = 3
        else:
          level = rLevel.index(r)
        data[0] = re.sub(r, "", data[0])
        break
    if 0 == level:
      self.tagStack = []
      self.tagStack.append((data[0], level, -1))
    elif 99 > level \
    and len(self.tagStack)>0:
      while True:
        last = self.tagStack[len(self.tagStack)-1]
        if last[1] < level:
          self.tagStack.append((data[0], level, last[1]))
          break
        else:
          self.tagStack.pop()
    if level>0 and len(self.tagStack)>1:
      last = self.tagStack[len(self.tagStack)-2][1]
      while True:
        dataparser.addTag(dataparser.rowTags, dataparser.currRow,
                          [self.tagStack[last][0]], dataparser.rowSpan)
        last = self.tagStack[last][2]
        if -1 == last: break
    return 99 != level
  
class TagLevel2(TagLevel):
  def check(self, dataparser, data):
    if dataparser.currCol > 0: return False
    
    rLevel = [u"([一|二|三|四|五|六|七|八|九|十]+、)", #level 0
              u"([0-9]+\.)", #level 1
              u"(\([0-9]+\))", #level 2
              ]
    level = 99
    for r in rLevel:
      m = re.match(r, data[0])
      if None != m:
        level = rLevel.index(r)
        data[0] = re.sub(r, "", data[0])
        break
    if 0 == level:
      self.tagStack = []
      self.tagStack.append((data[0], level, -1))
    elif 99 > level \
    and len(self.tagStack)>0:
      while True:
        last = self.tagStack[len(self.tagStack)-1]
        if last[1] < level:
          self.tagStack.append((data[0], level, last[1]))
          break
        else:
          self.tagStack.pop()
    if level>0 and len(self.tagStack)>1:
      last = self.tagStack[len(self.tagStack)-2][1]
      while True:
        dataparser.addTag(dataparser.rowTags, dataparser.currRow,
                          [self.tagStack[last][0]], dataparser.rowSpan)
        last = self.tagStack[last][2]
        if -1 == last: break
    return 99 != level

class TagLevel3(TagLevel):
  def check(self, dataparser, data):
    if dataparser.currCol > 0: return False
    
    rLevel = "#\D+"
    m = re.match(rLevel, data[0])
    if None != m \
    and len(self.tagStack)>0:
      dataparser.addTag(dataparser.rowTags, dataparser.currRow,
                        [self.tagStack[len(self.tagStack)-1]], dataparser.rowSpan)
    else:
      self.tagStack.append(data[0])
    return False

